package com.cnap.multiclouldmgmt.service.impl;

import cn.hutool.extra.spring.SpringUtil;
import com.alibaba.fastjson.JSONObject;
import com.cnap.multiclouldmgmt.model.AccountInfo;
import com.cnap.multiclouldmgmt.model.RepByPage;
import com.cnap.multiclouldmgmt.model.enums.*;
import com.cnap.multiclouldmgmt.model.gpu.GpuFlavorInfo;
import com.cnap.multiclouldmgmt.model.k8sModels.EventModel;
import com.cnap.multiclouldmgmt.model.k8sModels.K8sClusterBasicModel;
import com.cnap.multiclouldmgmt.model.k8sModels.K8sClusterModel;
import com.cnap.multiclouldmgmt.model.microSvc.MsgInfoModel;
import com.cnap.multiclouldmgmt.model.privatecloud.CloudVendorInfo;
import com.cnap.multiclouldmgmt.model.vm.CheckDelVmModel;
import com.cnap.multiclouldmgmt.model.vm.VmCreateModel;
import com.cnap.multiclouldmgmt.model.vm.VmModel;
import com.cnap.multiclouldmgmt.repository.cnap.MsgCenterInfoMapper;
import com.cnap.multiclouldmgmt.repository.master.*;
import com.cnap.multiclouldmgmt.service.*;
import com.cnap.multiclouldmgmt.util.CnapCloudMgmtConstants;
import com.cnap.multiclouldmgmt.util.DateUtils;
import com.cnap.multiclouldmgmt.util.TokenUtils;
import io.fabric8.kubernetes.api.model.events.v1beta1.Event;
import io.fabric8.kubernetes.client.*;
import io.fabric8.kubernetes.client.dsl.EventingAPIGroupDSL;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import java.time.LocalDateTime;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;

@Service("MultiCloudMgmtServiceImpl")
public class MultiCloudMgmtServiceImpl implements MultiCloudMgmtService {

    private final static Logger LOGGER = LoggerFactory.getLogger(MultiCloudMgmtServiceImpl.class);

    @Autowired
    private K8sClusterInfoMapper k8sClusterInfoMapper;

    @Autowired
    private CacheService cacheService;

    @Autowired
    private K8sClusterEventsMapper k8sClusterEventsMapper;

    @Autowired
    private DockerHarborService dockerHarborService;

    @Autowired
    private K8sClusterService k8sClusterService;

    @Autowired
    private CnapSshService cnapSshService;

    @Autowired
    private VmResourceService vmResourceService;

    @Autowired
    private CnapKeyValueMapper cnapKeyValueMapper;

    @Autowired
    private VmCreateInfoMapper vmCreateInfoMapper;

    @Autowired
    private MsgCenterInfoMapper msgCenterInfoMapper;

    @Autowired
    private FlavorInfoMapper flavorInfoMapper;

    //操作日志系统的url
    @Value("${logSys.url}")
    private String url;

    @Value("${center.k8sStatusOverTime}")
    private  String k8sStatusOverTimeSeconds;

    private static final String ROOT_NAME = "root";

    private static final String INFORM_K8S_CHANGE_URL
            = "http://cnap-schedule-mgmt-be-svc.default.svc.cluster.local:8080";

    /**
     *
     * @param k8sClusterModel
     * @throws Exception
     */
    @Override
    public void addK8sCluster(K8sClusterModel k8sClusterModel) throws Exception {

        //检查重名
        if (k8sClusterInfoMapper.getK8sClusterInfoByName(k8sClusterModel.getK8sClusterName()) != null){
            throw new Exception("集群名称冲突");
        }

        //检查ip冲突
        if (k8sClusterInfoMapper.getK8sClusterInfoByIp(k8sClusterModel.getK8sClusterIp()) != null){
            throw new Exception("集群ip已经存在");
        }


        //获取用户名
        if (StringUtils.isEmpty(k8sClusterModel.getCreateUser())) {
            String userName = TokenUtils.getUserName();
            k8sClusterModel.setCreateUser(userName);
        }
        //设定不同云的成本，目前写死
        if (StringUtils.equals(k8sClusterModel.getCloudVendor(),CloudVendorsEnum.TencentCloud.getDescribe())){

            k8sClusterModel.setCpuCost("10");
            k8sClusterModel.setCpuCostUnit("元/月");
            k8sClusterModel.setDiskCost("10");
            k8sClusterModel.setDiskCostUnit("元/GB/月");
            k8sClusterModel.setFileCost("10");
            k8sClusterModel.setFileCostUnit("元/GB/月");
            k8sClusterModel.setFpgaCost("10");
            k8sClusterModel.setGpuCost("10");
            k8sClusterModel.setGpuCostUnit("元/月");
            k8sClusterModel.setNetCost("10");
            k8sClusterModel.setNetCostUnit("元/Mbps/小时");
            k8sClusterModel.setNetCostType("固定带宽");
        }
        if (StringUtils.equals(k8sClusterModel.getCloudVendor(),CloudVendorsEnum.Aliyun.getDescribe())){

            k8sClusterModel.setCpuCost("100");
            k8sClusterModel.setCpuCostUnit("元/月");
            k8sClusterModel.setDiskCost("100");
            k8sClusterModel.setDiskCostUnit("元/GB/月");
            k8sClusterModel.setFileCost("100");
            k8sClusterModel.setFileCostUnit("元/GB/月");
            k8sClusterModel.setFpgaCost("100");
            k8sClusterModel.setGpuCost("100");
            k8sClusterModel.setGpuCostUnit("元/月");
            k8sClusterModel.setNetCost("100");
            k8sClusterModel.setNetCostUnit("元/Mbps/小时");
            k8sClusterModel.setNetCostType("固定带宽");
        }
        if (StringUtils.equals(k8sClusterModel.getCloudVendor(),CloudVendorsEnum.HuaWeiCloud.getDescribe())){

            k8sClusterModel.setCpuCost("1");
            k8sClusterModel.setCpuCostUnit("元/月");
            k8sClusterModel.setDiskCost("1");
            k8sClusterModel.setDiskCostUnit("元/GB/月");
            k8sClusterModel.setFileCost("1");
            k8sClusterModel.setFileCostUnit("元/GB/月");
            k8sClusterModel.setFpgaCost("1");
            k8sClusterModel.setGpuCost("1");
            k8sClusterModel.setGpuCostUnit("元/月");
            k8sClusterModel.setNetCost("1");
            k8sClusterModel.setNetCostUnit("元/Mbps/小时");
            k8sClusterModel.setNetCostType("固定带宽");
        }

        k8sClusterModel.setCreateTime(new Date());
        //初始化k8sclient
        initK8sClientPutCache(k8sClusterModel,k8sClusterModel.getK8sClusterConfig(),k8sClusterModel.getCreateUser());

        //保存前检查一下配额够不够
        if (!vmResourceService.checkUserK8sClusterQuota(k8sClusterModel.getCreateUser())) {

            throw new Exception("配额超出限制");
        }
        //保存到数据库
        k8sClusterInfoMapper.saveK8sClusterInfo(k8sClusterModel);

        //通知其他服务，有集群增加
        informOtherSvcClusterChange();

        vmResourceService.updateK8sClusterUseQuota(k8sClusterModel.getK8sClusterName(),k8sClusterModel.getCreateUser(),
                CnapCloudMgmtConstants.K8S_QUOTA_ADD,1);

    }

    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @return
     */
    @Override
    public int getK8sClusterNum(String cloudVendor,String k8sClusterName) {

        String userName = TokenUtils.getUserName();

        return k8sClusterInfoMapper.countK8sClusterByCloudVendorOrName(cloudVendor, k8sClusterName,userName,null);
    }

    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @param createUser
     * @return
     */
    @Override
    public int getK8sClusterNum(String cloudVendor, String k8sClusterName, String createUser) {
        return k8sClusterInfoMapper.countK8sClusterByCloudVendorOrName(cloudVendor, k8sClusterName,createUser,null);
    }

    @Override
    public int getK8sClusterNum(String cloudVendor, String k8sClusterName, String createUser,String accountName) {
        return k8sClusterInfoMapper.countK8sClusterByCloudVendorOrName(cloudVendor, k8sClusterName,createUser,accountName);
    }

    /**
     *
     * @param k8sClusterName
     * @return
     */
    @Override
    public String getK8sClusterIpByName(String k8sClusterName) {

        K8sClusterModel k8sClusterModel = k8sClusterInfoMapper.getK8sClusterInfoByName(k8sClusterName);
        if (k8sClusterModel == null) {
            return null;
        }

        return k8sClusterModel.getK8sClusterIp();
    }

    /**
     *
     * @param k8sClusterName
     * @return
     */
    @Override
    public K8sClusterBasicModel getK8sClusterInfoByName(String k8sClusterName) {
        
        K8sClusterModel k8sClusterModel = k8sClusterInfoMapper.getK8sClusterInfoByName(k8sClusterName);
        if (k8sClusterModel == null){
            return null;
        }

        K8sClusterBasicModel k8sClusterBasicModel = new K8sClusterBasicModel();
        k8sClusterBasicModel.setCloudVendor(k8sClusterModel.getCloudVendor());
        k8sClusterBasicModel.setK8sClusterName(k8sClusterModel.getK8sClusterName());
        k8sClusterBasicModel.setK8sClusterLocation(k8sClusterModel.getK8sClusterLocation());
        k8sClusterBasicModel.setK8sClusterIp(k8sClusterModel.getK8sClusterIp());
        k8sClusterBasicModel.setClusterInfoJson(k8sClusterModel.getClusterInfoJson());
        k8sClusterBasicModel.setCreateUser(k8sClusterModel.getCreateUser());
        k8sClusterBasicModel.setVmModels(k8sClusterModel.getVmModels());
        List<VmModel> vmModels = k8sClusterModel.getVmModels();
        if ((vmModels != null)&&(!vmModels.isEmpty())){
            k8sClusterBasicModel.setRegionId(vmModels.get(0).getRegionId());
            k8sClusterBasicModel.setZoneId(vmModels.get(0).getZoneId());
            k8sClusterBasicModel.setBusinessId(vmModels.get(0).getBusinessId());
            k8sClusterBasicModel.setVmModels(vmModels);
            k8sClusterBasicModel.setAccountName(vmModels.get(0).getAccountName());
        }
        try{

            initK8sClientPutCache(k8sClusterModel,
                    k8sClusterModel.getK8sClusterConfig(),k8sClusterModel.getCreateUser());
            KubernetesClient k8sClient = cacheService.getK8sClient(k8sClusterModel.getK8sClusterIp());
            //获取一下nodes信息，这样可以查询出到底有没有初始化成功
            checkK8sStatus(k8sClient);
            k8sClusterBasicModel.setK8sStatus(K8sStatusEnum.Running.getChineseDescribe());
            k8sClusterBasicModel.setK8sClient(k8sClient);

        }catch (Exception e){

            /**
             * （1）中心集群和故障集群之间网络连接错误
             * （2）集群资源使用过高
             * （3）集群token文件失效
             * （4）集群基础信息填写出错
             * （5）云基础设施异常
             * */
            k8sClusterBasicModel.setK8sStatus(K8sStatusEnum.Stop.getChineseDescribe());

            LOGGER.error("MultiCloudMgmtServiceImpl getK8sClusterInfoByName error k8sClusterIp={}",
                    k8sClusterModel.getK8sClusterIp(),e);
        }

        return k8sClusterBasicModel;
    }

    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @param pageSize
     * @param pageNo
     * @return
     */
    @Override
    public RepByPage<K8sClusterModel> getK8sClusterInfos(String cloudVendor, String k8sClusterName, int pageSize, int pageNo) {
        return getK8sClusterInfos(cloudVendor, k8sClusterName, pageSize, pageNo,null);
    }

    @Override
    public RepByPage<K8sClusterModel> getK8sClusterInfos(String cloudVendor, String k8sClusterName,
                                                         int pageSize, int pageNo, String createUser) {
        return getK8sClusterInfos(cloudVendor, k8sClusterName,pageSize,pageNo,createUser,null);
    }

    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @param pageSize
     * @param pageNo
     * @param createUser
     * @return
     */
    @Override
    public RepByPage<K8sClusterModel> getK8sClusterInfos(String cloudVendor,
                                                         String k8sClusterName, int pageSize, int pageNo,String createUser,
                                                         String accountName) {

        RepByPage<K8sClusterModel> result = new RepByPage<>();
        result.setPageSize(pageSize);
        result.setPageNo(pageNo);

        //如果是管理员，那就设置为空，返回所有
        if (StringUtils.equals(createUser,CnapCloudMgmtConstants.ADMIN_USER_NAME)){
            createUser = null;
        }

        int totalNum = getK8sClusterNum(cloudVendor, k8sClusterName,createUser,accountName);
        result.setTotalNum(totalNum);
        if (result.getTotalNum() != 0){


            result.setResult(new ArrayList<>());

            List<K8sClusterModel> tmp = k8sClusterInfoMapper.getK8sClusterByCloudVendorOrNameByPage(
                    cloudVendor,k8sClusterName,
                    pageSize,(pageNo - 1)*pageSize,createUser,accountName);


            //检查初始化，然后检查status
            for (K8sClusterModel k8sClusterModel:tmp){

                String blackList = cnapKeyValueMapper.getKvByKey("black");
                if (StringUtils.isNotEmpty(blackList)) {
                    if (StringUtils.contains(blackList, k8sClusterModel.getCloudVendor())) {
                        continue;
                    }
                }
                try{

                    initK8sClientPutCache(k8sClusterModel,
                            k8sClusterModel.getK8sClusterConfig(),k8sClusterModel.getCreateUser());
                    KubernetesClient k8sClient = cacheService.getK8sClient(k8sClusterModel.getK8sClusterIp());
                    //获取一下nodes信息，这样可以查询出到底有没有初始化成功
                    checkK8sStatus(k8sClient);

                    k8sClusterModel.setStatus(K8sStatusEnum.Running.getChineseDescribe());

                    k8sClusterModel.setInnerIp(k8sClusterService.getK8sClusterInnerIp(k8sClient));

                }catch (Exception e){

                    /**
                     * （1）中心集群和故障集群之间网络连接错误
                     * （2）集群资源使用过高
                     * （3）集群token文件失效
                     * （4）集群基础信息填写出错
                     * （5）云基础设施异常
                     */
                    k8sClusterModel.setStatus(K8sStatusEnum.Stop.getChineseDescribe());

                    LOGGER.error("MultiCloudMgmtServiceImpl get k8s status error k8sClusterIp={}",
                            k8sClusterModel.getK8sClusterIp(),e);
                }

                k8sClusterModel.setK8sClusterConfig(null);

                result.getResult().add(k8sClusterModel);
            }

        }else{
            result.setResult(new ArrayList<>());
        }

        return result;
    }

    /**
     *
     * @param k8sClusterModel
     * @throws Exception
     */
    @Override
    public void editK8sClusterInfo(K8sClusterModel k8sClusterModel) throws Exception {

        K8sClusterModel oldk8sClusterModel = k8sClusterInfoMapper.getK8sClusterInfoByIp(k8sClusterModel.getK8sClusterIp());
        if (oldk8sClusterModel == null){
            throw  new Exception("集群ip记录不存在");
        }

        K8sClusterModel k8sClusterModelByName
                = k8sClusterInfoMapper.getK8sClusterInfoByName(k8sClusterModel.getK8sClusterName());
        if ((k8sClusterModelByName != null)
                &&(!StringUtils.equals(k8sClusterModel.getK8sClusterIp(),k8sClusterModelByName.getK8sClusterIp()))){

            throw new Exception("集群名称已经存在");

        }

        k8sClusterInfoMapper.updateK8sClusterInfo(k8sClusterModel);

    }

    @Override
    public void deControlK8sCluster(String k8sClusterIp) {

        K8sClusterModel k8sClusterModel
                = k8sClusterInfoMapper.getK8sClusterInfoByIp(k8sClusterIp);

        k8sClusterInfoMapper.delK8sClusterByIp(k8sClusterIp);

        cacheService.clearK8sInfoCache(k8sClusterModel.getCreateUser(), k8sClusterModel.getK8sClusterName());

        vmResourceService.updateK8sClusterUseQuota(k8sClusterModel.getK8sClusterName(),
                k8sClusterModel.getCreateUser(),
                CnapCloudMgmtConstants.K8S_QUOTA_DEL,1);

        //删除扩缩容策略
        try {
            ScalingService scalingService = SpringUtil.getBean(ScalingService.class);
            scalingService.deleteByClusterName(k8sClusterModel.getK8sClusterName());
        }catch (Exception e){

            LOGGER.error("deControlK8sCluster scalingService fail k8sClusterIp={}",k8sClusterIp,e);
        }
        //删除缓存
        try {
            KubernetesClient k8sClient = cacheService.getK8sClient(k8sClusterIp);
            if (k8sClient != null) {
                cacheService.removeK8sClient(k8sClusterIp);
                k8sClient.close();
            }
        }catch (Exception e){

            LOGGER.error("delK8sCluster fail k8sClusterIp={}",k8sClusterIp,e);
        }


        //通知其他集群
        informOtherSvcClusterChange();

        //删除事件和告警
        try{
            k8sClusterEventsMapper.delEventsByIp(k8sClusterIp);

            //通知删除告警
            String delAlarmUrl = url + "/sys/alarm/delete/" + k8sClusterModel.getK8sClusterName() ;

            CloseableHttpClient httpclient = HttpClients.createDefault();

            HttpDelete httpDel = new HttpDelete(delAlarmUrl);

            RequestConfig requestConfig = RequestConfig.custom()
                    .setConnectTimeout(60 * 1000).setConnectionRequestTimeout(60 * 1000)
                    .setSocketTimeout(60 * 1000).build();
            httpDel.setConfig(requestConfig);

            CloseableHttpResponse response = httpclient.execute(httpDel);
            LOGGER.info("delK8sCluster alarm response code={}",response.getStatusLine().getStatusCode());

        }catch (Exception e){
            LOGGER.error("deControlK8sCluster del alarm and events fail k8sClusterIp={}",k8sClusterIp,e);
        }
    }

    /**
     *
     * @param k8sClusterIp
     */
    @Override
    public void delK8sCluster(String k8sClusterIp) throws Exception {

        K8sClusterModel k8sClusterModel
                = k8sClusterInfoMapper.getK8sClusterInfoByIp(k8sClusterIp);
        //检查有没有其他的应用部署着，有的话，就不能删除
        if (k8sClusterService.checkSvcExist(k8sClusterIp)){
            throw new Exception("集群上有未删除的服务");
        }

        if((k8sClusterModel.getVmModels() != null)&&(!k8sClusterModel.getVmModels().isEmpty())) {
            vmResourceService.delVms(k8sClusterModel.getVmModels());
        }

        k8sClusterInfoMapper.delK8sClusterByIp(k8sClusterIp);

        vmResourceService.updateK8sClusterUseQuota(k8sClusterModel.getK8sClusterName(),
                k8sClusterModel.getCreateUser(),
                CnapCloudMgmtConstants.K8S_QUOTA_DEL,1);

        cacheService.clearK8sInfoCache(k8sClusterModel.getCreateUser(), k8sClusterModel.getK8sClusterName());

        //删除扩缩容策略
        try {
            ScalingService scalingService = SpringUtil.getBean(ScalingService.class);
            scalingService.deleteByClusterName(k8sClusterModel.getK8sClusterName());
        }catch (Exception e){

            LOGGER.error("delK8sCluster scalingService fail k8sClusterIp={}",k8sClusterIp,e);
        }
        //删除缓存
        try {
            KubernetesClient k8sClient = cacheService.getK8sClient(k8sClusterIp);
            if (k8sClient != null) {
                cacheService.removeK8sClient(k8sClusterIp);
                k8sClient.close();
            }
        }catch (Exception e){

            LOGGER.error("delK8sCluster fail k8sClusterIp={}",k8sClusterIp,e);
        }

        //删除harbor策略
        //dockerHarborService.delHarborPolicy(k8sClusterIp);

        //通知其他集群
        informOtherSvcClusterChange();

        //删除事件和告警
        try{
            k8sClusterEventsMapper.delEventsByIp(k8sClusterIp);

            //通知删除告警
            String delAlarmUrl = url + "/sys/alarm/delete/" + k8sClusterModel.getK8sClusterName() ;

            CloseableHttpClient httpclient = HttpClients.createDefault();

            HttpDelete httpDel = new HttpDelete(delAlarmUrl);

            RequestConfig requestConfig = RequestConfig.custom()
                    .setConnectTimeout(60 * 1000).setConnectionRequestTimeout(60 * 1000)
                    .setSocketTimeout(60 * 1000).build();
            httpDel.setConfig(requestConfig);

            CloseableHttpResponse response = httpclient.execute(httpDel);
            LOGGER.info("delK8sCluster alarm response code={}",response.getStatusLine().getStatusCode());

        }catch (Exception e){
                LOGGER.error("delK8sCluster del alarm and events fail k8sClusterIp={}",k8sClusterIp,e);
        }
    }

    /**
     *
     * @param k8sClusterIp
     * @return
     * @throws Exception
     */
    @Override
    public K8sClusterBasicModel getK8sClientByIp(String k8sClusterIp) throws Exception {
        K8sClusterModel k8sClusterModel =
                k8sClusterInfoMapper.getK8sClusterInfoByIp(k8sClusterIp);
        if (k8sClusterModel == null){
            throw new Exception("该集群信息不存在");
        }
        K8sClusterBasicModel result = new K8sClusterBasicModel();
        try {
            initK8sClientPutCache(k8sClusterModel,
                        k8sClusterModel.getK8sClusterConfig(),k8sClusterModel.getCreateUser());
            KubernetesClient k8sClient = cacheService.getK8sClient(k8sClusterModel.getK8sClusterIp());
            checkK8sStatus(k8sClient);
            result.setK8sStatus(K8sStatusEnum.Running.getChineseDescribe());
            result.setK8sClient(k8sClient);
        }catch (Exception e){
                LOGGER.error("getK8sClientByIp init client fail k8sClusterIp={}",
                        k8sClusterModel.getK8sClusterIp(),e);
                if (!StringUtils.equals(k8sClusterModel.getCreateUser(),CnapCloudMgmtConstants.TEST_ADMIN)) {
                    throw new Exception("集群信息获取失败");
                }
            }

        result.setCloudVendor(k8sClusterModel.getCloudVendor());
        result.setK8sClusterIp(k8sClusterModel.getK8sClusterIp());
        result.setK8sClusterName(k8sClusterModel.getK8sClusterName());
        result.setK8sClusterLocation(k8sClusterModel.getK8sClusterLocation());
        result.setCreateUser(k8sClusterModel.getCreateUser());
        result.setZoneId(k8sClusterModel.getK8sClusterZone());
        if ((k8sClusterModel.getVmModels() != null)
                &&(!k8sClusterModel.getVmModels().isEmpty())){
            result.setZoneId(k8sClusterModel.getVmModels().get(0).getZoneId());
            result.setRegionId(k8sClusterModel.getVmModels().get(0).getRegionId());
            result.setBusinessId(k8sClusterModel.getVmModels().get(0).getBusinessId());
            result.setMasterNodeInnerIp(k8sClusterModel.getVmModels().get(0).getInnerIp());

            if (StringUtils.endsWithIgnoreCase(k8sClusterModel.getK8sClusterName(),CnapCloudMgmtConstants.ARCH_GPU)){
                GpuFlavorInfo gpuFlavorInfo = flavorInfoMapper.getGpuCloudFlavorInfo(k8sClusterModel.getAccountName(),
                        CloudVendorsEnum.changeToCloudeType(k8sClusterModel.getCloudVendor()),
                        result.getRegionId(),result.getZoneId(),
                        k8sClusterModel.getVmModels().get(0).getFlavorId());
                if (gpuFlavorInfo != null){
                    result.setGpuFlavorInfo(gpuFlavorInfo);
                }
            }
        }
        result.setVmModels(k8sClusterModel.getVmModels());

        return result;
    }

    @Override
    public List<K8sClusterBasicModel> getAllGpuInUseClusterInfos() {

        List<K8sClusterBasicModel> result = new ArrayList<>();
        List<K8sClusterModel> k8sClusterModels =
                k8sClusterInfoMapper.getAllGpuClusterInfo();
        for (K8sClusterModel k8sClusterModel:k8sClusterModels){
            K8sClusterBasicModel k8sClusterBasicModel = new K8sClusterBasicModel();

            try {
                initK8sClientPutCache(k8sClusterModel,
                        k8sClusterModel.getK8sClusterConfig(),k8sClusterModel.getCreateUser());
                KubernetesClient k8sClient = cacheService.getK8sClient(k8sClusterModel.getK8sClusterIp());
                checkK8sStatus(k8sClient);
                k8sClusterBasicModel.setK8sClient(k8sClient);

                k8sClusterBasicModel.setK8sStatus(K8sStatusEnum.Running.getChineseDescribe());
            }catch (Exception e){
                LOGGER.error("getK8sClientByCloudVendorAndName init client fail k8sClusterIp={}",
                        k8sClusterModel.getK8sClusterIp(),e);
                continue;
            }

            k8sClusterBasicModel.setCloudVendor(k8sClusterModel.getCloudVendor());
            k8sClusterBasicModel.setK8sClusterIp(k8sClusterModel.getK8sClusterIp());
            k8sClusterBasicModel.setK8sClusterName(k8sClusterModel.getK8sClusterName());
            k8sClusterBasicModel.setK8sClusterLocation(k8sClusterModel.getK8sClusterLocation());
            k8sClusterBasicModel.setCreateUser(k8sClusterModel.getCreateUser());
            if ((k8sClusterModel.getVmModels() != null)
                    &&(!k8sClusterModel.getVmModels().isEmpty())){
                k8sClusterBasicModel.setZoneId(k8sClusterModel.getVmModels().get(0).getZoneId());
                k8sClusterBasicModel.setRegionId(k8sClusterModel.getVmModels().get(0).getRegionId());
                k8sClusterBasicModel.setBusinessId(k8sClusterModel.getVmModels().get(0).getBusinessId());

                if (StringUtils.endsWithIgnoreCase(k8sClusterModel.getK8sClusterName(),CnapCloudMgmtConstants.ARCH_GPU)){
                    GpuFlavorInfo gpuFlavorInfo = flavorInfoMapper.getGpuCloudFlavorInfo(k8sClusterModel.getAccountName(),
                            CloudVendorsEnum.changeToCloudeType(k8sClusterModel.getCloudVendor()),
                            k8sClusterBasicModel.getRegionId(),k8sClusterBasicModel.getZoneId(),
                            k8sClusterModel.getVmModels().get(0).getFlavorId());
                    if (gpuFlavorInfo != null){
                        k8sClusterBasicModel.setGpuFlavorInfo(gpuFlavorInfo);
                    }
                }
            }

            result.add(k8sClusterBasicModel);
        }

        return result;
    }

    @Override
    public List<K8sClusterBasicModel> getK8sClientByCloudVendorAndName(String cloudVendor, String k8sClusterName, String accountName) throws Exception {
        List<K8sClusterModel> k8sClusterModels =
                k8sClusterInfoMapper.getK8sClusterInfoByNameAccount(k8sClusterName,accountName);
        List<K8sClusterBasicModel> result = new ArrayList<>();
        if ((k8sClusterModels == null)||(k8sClusterModels.isEmpty())){
            return result;
        }

        for (K8sClusterModel k8sClusterModel:k8sClusterModels) {
            K8sClusterBasicModel K8sClusterBasicModel = new K8sClusterBasicModel();

            try {
                initK8sClientPutCache(k8sClusterModel,
                        k8sClusterModel.getK8sClusterConfig(), k8sClusterModel.getCreateUser());
                KubernetesClient k8sClient = cacheService.getK8sClient(k8sClusterModel.getK8sClusterIp());
                checkK8sStatus(k8sClient);
                K8sClusterBasicModel.setK8sClient(k8sClient);

                K8sClusterBasicModel.setK8sStatus(K8sStatusEnum.Running.getChineseDescribe());
            } catch (Exception e) {
                LOGGER.error("getK8sClientByCloudVendorAndName init client fail k8sClusterIp={}",
                        k8sClusterModel.getK8sClusterIp(), e);
                continue;
            }

            K8sClusterBasicModel.setCloudVendor(k8sClusterModel.getCloudVendor());
            K8sClusterBasicModel.setK8sClusterIp(k8sClusterModel.getK8sClusterIp());
            K8sClusterBasicModel.setK8sClusterName(k8sClusterModel.getK8sClusterName());
            K8sClusterBasicModel.setK8sClusterLocation(k8sClusterModel.getK8sClusterLocation());
            K8sClusterBasicModel.setCreateUser(k8sClusterModel.getCreateUser());
            K8sClusterBasicModel.setVmModels(k8sClusterModel.getVmModels());
            if ((k8sClusterModel.getVmModels() != null)
                    && (!k8sClusterModel.getVmModels().isEmpty())) {
                K8sClusterBasicModel.setZoneId(k8sClusterModel.getVmModels().get(0).getZoneId());
                K8sClusterBasicModel.setRegionId(k8sClusterModel.getVmModels().get(0).getRegionId());
                K8sClusterBasicModel.setBusinessId(k8sClusterModel.getVmModels().get(0).getBusinessId());

                if (StringUtils.endsWithIgnoreCase(k8sClusterName, CnapCloudMgmtConstants.ARCH_GPU)) {
                    GpuFlavorInfo gpuFlavorInfo = flavorInfoMapper.getGpuCloudFlavorInfo(k8sClusterModel.getAccountName(),
                            CloudVendorsEnum.changeToCloudeType(k8sClusterModel.getCloudVendor()),
                            K8sClusterBasicModel.getRegionId(), K8sClusterBasicModel.getZoneId(),
                            k8sClusterModel.getVmModels().get(0).getFlavorId());
                    if (gpuFlavorInfo != null) {
                        K8sClusterBasicModel.setGpuFlavorInfo(gpuFlavorInfo);
                    }
                }
            }

            result.add(K8sClusterBasicModel);
        }

        return result;
    }

    /**
     *
     * @param cloudVendor
     * @param k8sClusterName
     * @return
     * @throws Exception
     */
    @Override
    public K8sClusterBasicModel getK8sClientByCloudVendorAndName(String cloudVendor, String k8sClusterName) throws Exception {

        K8sClusterModel k8sClusterModel =
                k8sClusterInfoMapper.getK8sClusterInfoByName(k8sClusterName);
        if (k8sClusterModel == null){
            throw new Exception("该集群信息不存在");
        }
        K8sClusterBasicModel result = new K8sClusterBasicModel();

        try {
                initK8sClientPutCache(k8sClusterModel,
                        k8sClusterModel.getK8sClusterConfig(),k8sClusterModel.getCreateUser());
                KubernetesClient k8sClient = cacheService.getK8sClient(k8sClusterModel.getK8sClusterIp());
                checkK8sStatus(k8sClient);
                result.setK8sClient(k8sClient);

                result.setK8sStatus(K8sStatusEnum.Running.getChineseDescribe());
            }catch (Exception e){
                LOGGER.error("getK8sClientByCloudVendorAndName init client fail k8sClusterIp={}",
                        k8sClusterModel.getK8sClusterIp(),e);
                throw new Exception("集群信息获取失败");
        }

        result.setCloudVendor(k8sClusterModel.getCloudVendor());
        result.setK8sClusterIp(k8sClusterModel.getK8sClusterIp());
        result.setK8sClusterName(k8sClusterModel.getK8sClusterName());
        result.setK8sClusterLocation(k8sClusterModel.getK8sClusterLocation());
        result.setCreateUser(k8sClusterModel.getCreateUser());
        result.setVmModels(k8sClusterModel.getVmModels());
        if ((k8sClusterModel.getVmModels() != null)
                &&(!k8sClusterModel.getVmModels().isEmpty())){
            result.setZoneId(k8sClusterModel.getVmModels().get(0).getZoneId());
            result.setRegionId(k8sClusterModel.getVmModels().get(0).getRegionId());
            result.setBusinessId(k8sClusterModel.getVmModels().get(0).getBusinessId());

            if (StringUtils.endsWithIgnoreCase(k8sClusterName,CnapCloudMgmtConstants.ARCH_GPU)){
                GpuFlavorInfo gpuFlavorInfo = flavorInfoMapper.getGpuCloudFlavorInfo(k8sClusterModel.getAccountName(),
                        CloudVendorsEnum.changeToCloudeType(k8sClusterModel.getCloudVendor()),
                        result.getRegionId(),result.getZoneId(),
                        k8sClusterModel.getVmModels().get(0).getFlavorId());
                if (gpuFlavorInfo != null){
                    result.setGpuFlavorInfo(gpuFlavorInfo);
                }
            }
        }

        return result;
    }

    /**
     *
     * @param creatUser 用户
     * @return
     */
    @Override
    public List<String> getCloudVendorsByCreateUser(String creatUser) {
        return k8sClusterInfoMapper.getClouudVendorsByCreateUser(creatUser);
    }

    /**
     *
     * @param userName
     * @return
     */
    @Override
    public List<String> getK8sIpsByUserName(String userName) {
        return k8sClusterInfoMapper.getK8sIpsByUserName(userName);
    }

    /**
     *
     * @return
     */
    @Override
    public List<String> getAllK8sClusterIps() {
        return k8sClusterInfoMapper.getAllIps();
    }

    /**
     *
     * @return
     */
    @Override
    public List<String> getAllK8sClusterNames() {
        return k8sClusterInfoMapper.getAllNames();
    }

    /**
     *
     * @param k8sClusterModel
     * @param configCotent
     * @param userName
     */
    private void initK8sClientPutCache(K8sClusterModel k8sClusterModel, String configCotent, String userName){

        if (cacheService.getK8sClient(k8sClusterModel.getK8sClusterIp()) != null){
            return;
        }

        Config config= new ConfigBuilder()
                .withMasterUrl("https://" + k8sClusterModel.getK8sClusterIp() + ":6443")
                .withTrustCerts(true).withOauthToken(configCotent)
                .withConnectionTimeout(5*60*1000).build();


        KubernetesClient k8sClient = new KubernetesClientBuilder().withConfig(config).build();
        
        //放入到缓存中
        cacheService.putK8sClient(k8sClusterModel.getK8sClusterIp(),k8sClient);

        //注册事件监控
        addK8sClientEventWatch(k8sClusterModel, k8sClient,userName);

    }

    private void addK8sClientEventWatch(K8sClusterModel k8sClusterModel,KubernetesClient k8sClient,String userName){

        EventingAPIGroupDSL eventingAPIGroupDSL = k8sClient.events();
        if (eventingAPIGroupDSL.v1beta1().events() == null){

            return;
        }

        if (StringUtils.equals(userName,CnapCloudMgmtConstants.TEST_ADMIN)){
            return;
        }
        eventingAPIGroupDSL.v1beta1().events().inAnyNamespace().watch(new Watcher<Event>() {

            @Override
            public void eventReceived(Action action, Event resource) {

                try {
                    EventModel eventModel = transResourceToEventModel(resource,k8sClusterModel,userName);
                    //放入数据库
                    putK8sEventToDb(eventModel);
                }catch (Throwable e){
                    LOGGER.error("addK8sClientEventWatch eventReceived fail ip={}",k8sClusterModel.getK8sClusterIp(),e);
                }
            }

            @Override
            public void onClose(WatcherException cause) {}
        });
    }

    @Override
    public EventModel transResourceToEventModel(Event resource,K8sClusterModel k8sClusterModel,String userName) {

        EventModel eventModel = new EventModel();
        eventModel.setEventType(resource.getType());
        eventModel.setInvolvedObj(resource.getRegarding().getKind()
                + ":" + resource.getRegarding().getName());
        eventModel.setMsg(resource.getNote());
        eventModel.setNamespace(resource.getMetadata().getNamespace());
        eventModel.setResource(resource.getDeprecatedSource().getComponent()
                + resource.getDeprecatedSource().getHost());
        eventModel.setCreateTime(LocalDateTime.now());
        eventModel.setNote(resource.getNote());
        eventModel.setReason(resource.getReason());
        eventModel.setK8sClusterIp(k8sClusterModel.getK8sClusterIp());
        eventModel.setCloudVendor(k8sClusterModel.getCloudVendor());
        eventModel.setK8sClusterName(k8sClusterModel.getK8sClusterName());
        eventModel.setK8sClusterUserName(userName);

        return eventModel;
    }

    /**
     *
     * @param eventModel
     */
    @Override
    public void putK8sEventToDb(EventModel eventModel) {

        k8sClusterEventsMapper.addK8sClusterEvent(eventModel);
    }

    @Override
    public K8sClusterModel getK8sClusterModelByVmCreateModel(VmCreateModel vmCreateModel){

        if ((vmCreateModel.getVmModels() == null)||(vmCreateModel.getVmModels().isEmpty())){
            return null;
        }

        VmModel firstVmModel = vmCreateModel.getVmModels().get(0);
        K8sClusterModel k8sClusterModel =
                k8sClusterInfoMapper.getK8sClusterInfoByZone(
                        getZoneId(firstVmModel,vmCreateModel.getUserName()));
        if (k8sClusterModel == null){

            k8sClusterModel = new K8sClusterModel();
            k8sClusterModel.setCreateTime(new Date());
            k8sClusterModel.setCloudVendor(CloudVendorsEnum.getDescribeByCloudeType(firstVmModel.getCloudType()));
            k8sClusterModel.setCreateUser(vmCreateModel.getUserName());
            k8sClusterModel.setK8sClusterIp(firstVmModel.getExternalIp());
            k8sClusterModel.setK8sClusterName(k8sClusterModel.getCloudVendor() + firstVmModel.getName());
            k8sClusterModel.setK8sClusterZone(getZoneId(firstVmModel, vmCreateModel.getUserName()));
        }

        return k8sClusterModel;
    }

    @Override
    public String installK8sFramwork(VmCreateModel vmCreateModel) throws Exception {

        if ((vmCreateModel.getVmModels() == null)
                ||(vmCreateModel.getVmModels().isEmpty())){
            return null;
        }

        //有时候刚开通的机器，后台没有准备好，设定一个等待时间

        VmModel firstVmModel = vmCreateModel.getVmModels().get(0);
        //根据创建的机器，尝试获取有没有已经建立的集群
        List<K8sClusterModel> k8sClusterModels =
                k8sClusterInfoMapper.getK8sClusterInfosByZone(
                        getZoneIdWithBizId(firstVmModel,vmCreateModel.getUserName()));
        //如果强制隔离，就建一个新的集群
        if (vmCreateModel.isForcedIsolation()){
            k8sClusterModels = null;
        }

        K8sClusterModel oldK8sClusterModel = new K8sClusterModel();
        //不为null 说明已经安装了，只要给这些vm装worker就行了
        if ((k8sClusterModels != null)&&(!k8sClusterModels.isEmpty())){

            //说明只剩下安装k8s等步骤，通知其他系统
            updateInstallWorkerMsg(vmCreateModel);


            /**
             * 找到那个原始的集群
             * 根据bizId判断
             */
            for (K8sClusterModel k8sClusterModel:k8sClusterModels){
                if (StringUtils.equals(firstVmModel.getBusinessId(),
                        k8sClusterModel.getVmModels().get(0).getBusinessId())){
                    oldK8sClusterModel = k8sClusterModel;
                }
            }

            LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
            LOGGER.warn("installK8sFramwork k8s exist ip={}",k8sClusterModels.get(0).getK8sClusterIp());
            LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~");

            //找到原有master节点
            VmModel masterModel = null;
            for (VmModel vmModel : oldK8sClusterModel.getVmModels()){
                if (StringUtils.equals(vmModel.getK8sCluterRole(), CnapCloudMgmtConstants.CNAP_VM_MASTER)){
                    masterModel = vmModel;
                    break;
                }
            }

            for (VmModel vmModel:vmCreateModel.getVmModels()){

                //添加worker节点，给worker节点安装prometheus
                installK8sWorker(oldK8sClusterModel.getK8sClusterName(),vmModel,
                        masterModel,vmModel.getCategory(),vmCreateModel);
                vmModel.setK8sCluterRole(CnapCloudMgmtConstants.CNAP_VM_WORKER);
                vmModel.setCreateTimeStamp((new Date()).getTime());
                oldK8sClusterModel.getVmModels().add(vmModel);

                //暂停15秒，防止没有创建成功
                try{
                    Thread.sleep(15*1000L);
                }catch(InterruptedException e){
                    //防止线程异常，暂停失败
                    LOGGER.error("installK8sFramwork after installK8sWorker sleep fail e",e);
                    Thread.currentThread().interrupt();
                }catch (Exception e) {
                    LOGGER.error("installK8sFramwork err. ",e);
                }

                /**
                 * 增加节点的label
                 * 防止增加label失败影响整个流程
                 */
                if (StringUtils.isNotEmpty(vmModel.getCategory())) {
                    String tmpNodeName = CnapCloudMgmtConstants.CNAP_NODE_PREFIX + vmModel.getInnerIp();
                    String labelKey = CnapCloudMgmtConstants.NODE_APP_TYPE_LABEL + vmModel.getCategory().toLowerCase();
                    String labelValue = vmModel.getCategory();
                    try {
                        K8sClusterBasicModel k8sClusterBasicModel = null;
                        if (StringUtils.equals(vmCreateModel.getUserName(), CnapCloudMgmtConstants.TEST_ADMIN)) {
                            k8sClusterBasicModel = new K8sClusterBasicModel();

                        } else {
                            k8sClusterBasicModel = getK8sClusterInfoByName(oldK8sClusterModel.getK8sClusterName());
                        }

                        k8sClusterService.addNodeLabel(k8sClusterBasicModel, tmpNodeName,
                                labelKey, labelValue);

                        //装svc的label，用于选择亲和性
                        String svcName = vmCreateModel.getSvcName();
                        if (StringUtils.isEmpty(svcName)){
                            svcName = vmModel.getSvcNameLabelValue();
                        }

                        if (StringUtils.isNotEmpty(svcName)) {
                            k8sClusterService.addNodeLabel(k8sClusterBasicModel, tmpNodeName,
                                    CnapCloudMgmtConstants.NODE_APP_LABEL,
                                    svcName);

                            k8sClusterService.addNodeLabel(k8sClusterBasicModel, tmpNodeName,
                                    CnapCloudMgmtConstants.NODE_APP_LABEL_ID,
                                    getRealSvcId(vmCreateModel.getSvcId()));
                        }else {
                            //这种情况可能是已有虚机加入已有集群，那就给他打上empty标签
                            k8sClusterService.addNodeLabel(k8sClusterBasicModel, tmpNodeName,
                                    CnapCloudMgmtConstants.NODE_APP_LABEL,CnapCloudMgmtConstants.NODE_APP_LABEL_EMPTY);
                        }

                        //v2.0. 如果是扩容，写入msg
                        if (StringUtils.containsIgnoreCase(vmCreateModel.getSvcId(),
                                CnapCloudMgmtConstants.SCALE_SVC_PRE)){
                            MsgInfoModel msgInfoModel = new MsgInfoModel();
                            msgInfoModel.setCreateTime(new Date());
                            msgInfoModel.setUserName(k8sClusterBasicModel.getCreateUser());
                            msgInfoModel.setItem01("动态扩缩容");
                            msgInfoModel.setItem02(k8sClusterBasicModel.getK8sClusterName());
                            msgInfoModel.setItem03(DateUtils.changeDateFormatYYYYMMDDHHMMSS(msgInfoModel.getCreateTime()));
                            msgInfoModel.setMsgType(MsgTypeEnum.ResDeploySuccess.getMsgType());
                            msgCenterInfoMapper.saveMsgInfo(msgInfoModel);
                        }

                    } catch (Exception e) {
                        LOGGER.error("installK8sFramwork addNodeLabel work node fail clusterName={} nodeName={} labelkey={}" +
                                        " labeValue={} ",
                                oldK8sClusterModel.getK8sClusterName(), tmpNodeName, labelKey, labelValue,
                                e);
                    }
                }

                //刷新一下vmJson
                try {
                    K8sClusterModel tmpK8sClusterModel =
                            k8sClusterInfoMapper.getK8sClusterInfoByName(oldK8sClusterModel.getK8sClusterName());
                    if (tmpK8sClusterModel != null){
                        oldK8sClusterModel = tmpK8sClusterModel;
                    }
                    //先读后写入，防止多线程异常，防止多线程异常
                    oldK8sClusterModel.getVmModels().add(vmModel);
                    oldK8sClusterModel.setVmInfoJson(JSONObject.toJSONString(oldK8sClusterModel.getVmModels()));
                    k8sClusterInfoMapper
                            .updateClusterVmInfo(oldK8sClusterModel.getK8sClusterIp(), oldK8sClusterModel.getVmInfoJson());
                }catch (Exception e){
                    LOGGER.error("installK8sFramwork add worker json fail ip={} ",oldK8sClusterModel.getK8sClusterIp(),e);
                }

                //安装注入工具
                cnapSshService.copyAndInstallInjectTools(masterModel,vmModel);
            }

            //到这一步，说明k8s安装完成
            //改状态，通知其他资源
            vmCreateModel.setType(StageTypeEnum.K8S.getType());
            vmCreateModel.setStepOrder(StageDetailTypeEnum.THREE_SIX.getStepOrder());
            vmCreateModel.setDetailType(StageDetailTypeEnum.THREE_SIX.getDetailType());
            vmCreateModel.setOrder(StageDetailTypeEnum.THREE_SIX.getOrder());

            VmCreateModel vmSendCreateModel36 = new VmCreateModel();
            vmResourceService.initVmSendCreateModel(vmSendCreateModel36, vmCreateModel);
            vmResourceService.updateStageMsg(vmSendCreateModel36);
            vmCreateInfoMapper.updateVmCreateInfo(vmCreateModel);
            //结束通知

            //改状态，通知其他资源
            vmCreateModel.setType(StageTypeEnum.K8S.getType());
            vmCreateModel.setStepOrder(StageDetailTypeEnum.THREE_SEVEN.getStepOrder());
            vmCreateModel.setDetailType(StageDetailTypeEnum.THREE_SEVEN.getDetailType());
            vmCreateModel.setOrder(StageDetailTypeEnum.THREE_SEVEN.getOrder());

            VmCreateModel vmSendCreateModel37 = new VmCreateModel();
            List<Map<String,String>> extranInfo = new ArrayList<>();
            for (VmModel vmModel:vmCreateModel.getVmModels()){
                Map<String,String> param = new HashMap<>();
                param.put("name",oldK8sClusterModel.getK8sClusterName());
                param.put("providerName",CloudVendorsEnum.changeToCloudeType(oldK8sClusterModel.getCloudVendor()));
                param.put("ip",vmModel.getExternalIp());
                param.put("innerIp",vmModel.getInnerIp());
                extranInfo.add(param);
            }
            vmSendCreateModel37.setExtraInfo(extranInfo);
            vmResourceService.initVmSendCreateModel(vmSendCreateModel37,vmCreateModel);
            vmResourceService.updateStageMsg(vmSendCreateModel37);
            vmCreateInfoMapper.updateVmCreateInfo(vmCreateModel);
            //如果有等他结束的其他svc，改成状态0，可以继续开始了
            vmCreateInfoMapper.updateWaitCreateInfo0(vmCreateModel.getSvcId());
            //结束通知

            //null就不需要安装agent
            return null;

        }else {

            //要给第一个装master，剩下的装worker
            VmModel masterModel = vmCreateModel.getVmModels().get(0);

            LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
            LOGGER.warn("installK8sFramwork install k8s ip={}",masterModel.getExternalIp());

            //正式安装k8s,安装完成之后，获取config
            String configContet = installK8sMaster(vmCreateModel);
            //给节点设置role
            masterModel.setK8sCluterRole(CnapCloudMgmtConstants.CNAP_VM_MASTER);

            //注册集群要用的信息
            K8sClusterModel k8sClusterModel = new K8sClusterModel();
            k8sClusterModel.setCreateTime(new Date());
            k8sClusterModel.setK8sClusterConfig(configContet);
            k8sClusterModel.setCloudVendor(CloudVendorsEnum.getDescribeByCloudeType(masterModel.getCloudType()));
            k8sClusterModel.setCreateUser(vmCreateModel.getUserName());
            k8sClusterModel.setK8sClusterIp(masterModel.getExternalIp());
            k8sClusterModel.setRegionId(masterModel.getRegionId());
            k8sClusterModel.setZoneId(masterModel.getZoneId());
            k8sClusterModel.setAccountName(vmCreateModel.getAccountName());
            //如果是应用集群扩容引发的创建，就已经指定了clusterName了，那就需要强制指定名字
            if(StringUtils.isNotEmpty(masterModel.getAddclusterName())){

                k8sClusterModel.setK8sClusterName(masterModel.getAddclusterName());

            }else {

                k8sClusterModel.setK8sClusterName(k8sClusterModel.getCloudVendor() + masterModel.getName()+masterModel.getArch());
            }

            if (vmCreateModel.isForcedIsolation()){
                //独占的要在zone上多加个字段
                k8sClusterModel.setK8sClusterZone(getZoneIdWithBizId(masterModel,
                        vmCreateModel.getUserName()+CnapCloudMgmtConstants.FORCED_ISOLATION));
                k8sClusterModel.setK8sClusterName(k8sClusterModel.getK8sClusterName() + CnapCloudMgmtConstants.FORCED_ISOLATION);


            }else {

                k8sClusterModel.setK8sClusterZone(getZoneIdWithBizId(masterModel, vmCreateModel.getUserName()));
            }
            k8sClusterModel.setVmModels(new ArrayList<>());
            //给vm设置role
            for (VmModel vmModel:vmCreateModel.getVmModels()){

                if (StringUtils.isEmpty(vmModel.getK8sCluterRole())) {
                        vmModel.setK8sCluterRole(CnapCloudMgmtConstants.CNAP_VM_WORKER);
                }
                vmModel.setCreateTimeStamp((new Date()).getTime());
                k8sClusterModel.getVmModels().add(vmModel);
            }

            LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
            LOGGER.warn("installK8sMaster add k8sCluster k8s ip={}",masterModel.getExternalIp());

            //安装集群
            addK8sCluster(k8sClusterModel);
            cnapSshService.copyAndInstallInjectTools(null,masterModel);

            //暂停15秒，防止没有创建成功
            try{
                Thread.sleep(15*1000L);
            } catch(InterruptedException e){
                //防止线程异常，暂停失败
                LOGGER.error("installK8sFramwork after addK8sCluster sleep fail e",e);
                Thread.currentThread().interrupt();
            }catch (Exception e) {
                LOGGER.error("installK8sFramwork after addK8sCluster sleep err. ",e);
            }

            //给节点加标签
            for (VmModel vmModel:vmCreateModel.getVmModels()) {

                //master节点特殊处理,master 节点，两个label都有
                if (StringUtils.equals(vmModel.getK8sCluterRole(),CnapCloudMgmtConstants.CNAP_VM_MASTER)){

                    String tmpNodeName = CnapCloudMgmtConstants.CNAP_NODE_PREFIX + vmModel.getInnerIp();
                    try {

                        k8sClusterService.addNodeLabel(k8sClusterModel.getK8sClusterName(),
                                tmpNodeName,
                                CnapCloudMgmtConstants.NODE_APP_TYPE_LABEL + "ai",
                                "ai");
                        k8sClusterService.addNodeLabel(k8sClusterModel.getK8sClusterName(),
                                tmpNodeName,
                                CnapCloudMgmtConstants.NODE_APP_TYPE_LABEL +"bigdata",
                                "bigdata");
                        //装svc的label，用于选择亲和性
                        String svcName = vmCreateModel.getSvcName();
                        if (StringUtils.isEmpty(svcName)){
                            svcName = vmModel.getSvcNameLabelValue();
                        }
                        if (StringUtils.isNotEmpty(svcName)) {
                            k8sClusterService.addNodeLabel(k8sClusterModel.getK8sClusterName(), tmpNodeName,
                                    CnapCloudMgmtConstants.NODE_APP_LABEL,
                                    svcName);
                            k8sClusterService.addNodeLabel(k8sClusterModel.getK8sClusterName(), tmpNodeName,
                                    CnapCloudMgmtConstants.NODE_APP_LABEL_ID,
                                    getRealSvcId(vmCreateModel.getSvcId()));
                        }else {
                            //这种情况可能是已有虚机加入已有集群，那就给他打上empty标签
                            k8sClusterService.addNodeLabel(k8sClusterModel.getK8sClusterName(), tmpNodeName,
                                    CnapCloudMgmtConstants.NODE_APP_LABEL,CnapCloudMgmtConstants.NODE_APP_LABEL_EMPTY);
                        }
                    }catch (Exception e){
                        LOGGER.error("installK8sFramwork addNodeLabel fail clusterName={} nodeName={}",
                                k8sClusterModel.getK8sClusterName(),tmpNodeName, e);
                    }

                    continue;
                }

                //走到这都是worker节点
                cnapSshService.copyAndInstallInjectTools(masterModel,vmModel);

                if (StringUtils.isNotEmpty(vmModel.getCategory())) {
                    //防止节点加label失败影响整个流程
                    String tmpNodeName = CnapCloudMgmtConstants.CNAP_NODE_PREFIX + vmModel.getInnerIp();
                    String labelKey =  CnapCloudMgmtConstants.NODE_APP_TYPE_LABEL + vmModel.getCategory().toLowerCase();
                    String labelValue = vmModel.getCategory();
                    try {
                        k8sClusterService.addNodeLabel(k8sClusterModel.getK8sClusterName(),
                                tmpNodeName,
                                labelKey,labelValue);
                        //装svc的label，用于选择亲和性
                        String svcName = vmCreateModel.getSvcName();
                        if (StringUtils.isEmpty(svcName)){
                            svcName = vmModel.getSvcNameLabelValue();
                        }
                        if (StringUtils.isNotEmpty(svcName)) {
                            k8sClusterService.addNodeLabel(k8sClusterModel.getK8sClusterName(), tmpNodeName,
                                    CnapCloudMgmtConstants.NODE_APP_LABEL,
                                    svcName);
                            k8sClusterService.addNodeLabel(k8sClusterModel.getK8sClusterName(), tmpNodeName,
                                    CnapCloudMgmtConstants.NODE_APP_LABEL_ID,
                                    getRealSvcId(vmCreateModel.getSvcId()));
                        }else {
                            //这种情况可能是已有虚机加入已有集群，那就给他打上empty标签
                            k8sClusterService.addNodeLabel(k8sClusterModel.getK8sClusterName(), tmpNodeName,
                                    CnapCloudMgmtConstants.NODE_APP_LABEL,CnapCloudMgmtConstants.NODE_APP_LABEL_EMPTY);
                        }

                    }catch (Exception e){
                        LOGGER.error("installK8sFramwork addNodeLabel fail clusterName={} nodeName={} labelkey={}" +
                                        " labeValue={} ",
                                k8sClusterModel.getK8sClusterName(),tmpNodeName,labelKey,labelValue,
                                e);
                    }
                }
            }

            return masterModel.getExternalIp();
        }
    }

    //处理返回svcId
    private String getRealSvcId(String svcId){

        if (StringUtils.contains(svcId,CnapCloudMgmtConstants.SCALE_SVC_PRE)){

            return svcId.split(CnapCloudMgmtConstants.SCALE_SVC_PRE)[1];
        }

        if (StringUtils.contains(svcId,CnapCloudMgmtConstants.SCALE_CLUSTER_PRE)){

            return svcId.split(CnapCloudMgmtConstants.SCALE_CLUSTER_PRE)[1];
        }

        if (StringUtils.contains(svcId,CnapCloudMgmtConstants.MIGRATE_VM)){

            return svcId.split(CnapCloudMgmtConstants.MIGRATE_VM)[1];
        }

        return svcId;
    }

    //扩容时候通知其他系统，从3-2到3-5，只剩下安装k8s和标签
    private void updateInstallWorkerMsg(VmCreateModel vmCreateModel) {

        vmCreateModel.setType(StageTypeEnum.K8S.getType());
        vmCreateModel.setStepOrder(StageDetailTypeEnum.THREE_TOW.getStepOrder());
        vmCreateModel.setDetailType(StageDetailTypeEnum.THREE_TOW.getDetailType());
        vmCreateModel.setOrder(StageDetailTypeEnum.THREE_TOW.getOrder());

        VmCreateModel vmSendCreateModel32 = new VmCreateModel();
        vmResourceService.initVmSendCreateModel(vmSendCreateModel32, vmCreateModel);
        vmResourceService.updateStageMsg(vmSendCreateModel32);

        vmCreateModel.setType(StageTypeEnum.K8S.getType());
        vmCreateModel.setStepOrder(StageDetailTypeEnum.THREE_THREE.getStepOrder());
        vmCreateModel.setDetailType(StageDetailTypeEnum.THREE_THREE.getDetailType());
        vmCreateModel.setOrder(StageDetailTypeEnum.THREE_THREE.getOrder());

        VmCreateModel vmSendCreateModel33 = new VmCreateModel();
        vmResourceService.initVmSendCreateModel(vmSendCreateModel33, vmCreateModel);
        vmResourceService.updateStageMsg(vmSendCreateModel33);

        vmCreateModel.setType(StageTypeEnum.K8S.getType());
        vmCreateModel.setStepOrder(StageDetailTypeEnum.THREE_FOUR.getStepOrder());
        vmCreateModel.setDetailType(StageDetailTypeEnum.THREE_FOUR.getDetailType());
        vmCreateModel.setOrder(StageDetailTypeEnum.THREE_FOUR.getOrder());

        VmCreateModel vmSendCreateModel34 = new VmCreateModel();
        vmResourceService.initVmSendCreateModel(vmSendCreateModel34, vmCreateModel);
        vmResourceService.updateStageMsg(vmSendCreateModel34);

        vmCreateModel.setType(StageTypeEnum.K8S.getType());
        vmCreateModel.setStepOrder(StageDetailTypeEnum.THREE_FIVE.getStepOrder());
        vmCreateModel.setDetailType(StageDetailTypeEnum.THREE_FIVE.getDetailType());
        vmCreateModel.setOrder(StageDetailTypeEnum.THREE_FIVE.getOrder());

        VmCreateModel vmSendCreateModel35 = new VmCreateModel();
        vmResourceService.initVmSendCreateModel(vmSendCreateModel35, vmCreateModel);
        vmResourceService.updateStageMsg(vmSendCreateModel35);

        vmCreateInfoMapper.updateVmCreateInfo(vmCreateModel);
    }

    //返回configcontent
    private String installK8sMaster(VmCreateModel vmCreateModel) throws Exception {

        VmModel masterModel = null;
        List<VmModel> workerModels = new ArrayList<>();
        for (VmModel vmModel:vmCreateModel.getVmModels()){

            if (masterModel == null){
                masterModel = vmModel;
                continue;
            }
            workerModels.add(vmModel);
        }

        //腾讯云特殊处理，如果是腾讯云，那就要设置root用户登陆权限
        if (StringUtils.equals(masterModel.getCloudType(),CloudVendorsEnum.TencentCloud.getCloudType())){
            cnapSshService.setTencentRootSsh(masterModel);
        }

        LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
        LOGGER.warn("installK8sMaster begin copy master k8s ip={}",masterModel.getExternalIp());
        //如果是重试，跳过这个步骤
        if (StageDetailTypeEnum.THREE_TOW.getStepOrder() > vmCreateModel.getStepOrder()) {
            if (!cnapSshService.getPackageNormal(masterModel)) {

                //记录开始拷贝的时间
                long copyPackageTimeBegin = (new Date()).getTime();

                //拷贝安装包去节点
                cnapSshService.copyMasterInstallPackage(masterModel);

                //到这一步，说明开始拷贝资源
                //改状态，通知其他资源
                vmCreateModel.setType(StageTypeEnum.K8S.getType());
                vmCreateModel.setStepOrder(StageDetailTypeEnum.THREE_TOW.getStepOrder());
                vmCreateModel.setDetailType(StageDetailTypeEnum.THREE_TOW.getDetailType());
                vmCreateModel.setOrder(StageDetailTypeEnum.THREE_TOW.getOrder());

                VmCreateModel vmSendCreateModel32 = new VmCreateModel();
                vmResourceService.initVmSendCreateModel(vmSendCreateModel32, vmCreateModel);
                vmResourceService.updateStageMsg(vmSendCreateModel32);
                vmCreateInfoMapper.updateVmCreateInfo(vmCreateModel);
                //结束通知

                /**
                 * 检查包有没有copy好
                 */
                int i = 0;
                while (true) {

                    //拷贝完成就跳出
                    if (cnapSshService.getPackageNormal(masterModel)) {
                        break;
                    }

                    try {
                        Thread.sleep(3 * 60 * 1000L);
                    } catch (InterruptedException e) {
                        //防止线程异常，暂停失败
                        LOGGER.error("getPackageNormal sleep fail e", e);
                        Thread.currentThread().interrupt();
                    } catch (Exception e) {
                        LOGGER.error("getPackageNormal sleep err. ", e);
                    }
                    i++;
                    if (i > 100) {
                        break;
                    }
                    //给ut跳出用
                    if (StringUtils.equals(vmCreateModel.getUserName(), CnapCloudMgmtConstants.TEST_ADMIN)) {
                        break;
                    }
                }

                long copyPackageTimeEnd = (new Date()).getTime();
                String copyTimeMin = cnapKeyValueMapper.getKvByKey("copyTimeMin");
                if (StringUtils.isNotEmpty(copyTimeMin)){
                    long minWaitTime = Long.parseLong(copyTimeMin);
                    long copyPackageCostTime = copyPackageTimeEnd - copyPackageTimeBegin;
                    LOGGER.warn("install process copy package costTime={} minWaitTime={}",copyPackageCostTime,minWaitTime);
                    if (copyPackageCostTime < minWaitTime){
                        try {
                            Thread.sleep((minWaitTime - copyPackageCostTime));
                        } catch (InterruptedException e) {
                            //防止线程异常，暂停失败
                            LOGGER.error("wait min time sleep fail e", e);
                            Thread.currentThread().interrupt();
                        } catch (Exception e) {
                            LOGGER.error("wait min time slee sleep err. ", e);
                        }
                    }

                }
            }
        }

        if (StageDetailTypeEnum.THREE_THREE.getStepOrder() > vmCreateModel.getStepOrder()) {
            LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
            LOGGER.warn("installK8sMaster begin tar package k8s ip={}", masterModel.getExternalIp());
            //解压缩安装包
            cnapSshService.tarXvfInstallPackage(masterModel);

            //到这一步，说明开始确认拷贝环境
            //改状态，通知其他资源
            vmCreateModel.setType(StageTypeEnum.K8S.getType());
            vmCreateModel.setStepOrder(StageDetailTypeEnum.THREE_THREE.getStepOrder());
            vmCreateModel.setDetailType(StageDetailTypeEnum.THREE_THREE.getDetailType());
            vmCreateModel.setOrder(StageDetailTypeEnum.THREE_THREE.getOrder());

            VmCreateModel vmSendCreateModel33 = new VmCreateModel();
            vmResourceService.initVmSendCreateModel(vmSendCreateModel33, vmCreateModel);
            vmResourceService.updateStageMsg(vmSendCreateModel33);
            vmCreateInfoMapper.updateVmCreateInfo(vmCreateModel);
            //结束通知
        }

        if(StageDetailTypeEnum.THREE_FOUR.getStepOrder() > vmCreateModel.getStepOrder()) {
            LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
            LOGGER.warn("installK8sMaster begin install tools k8s ip={}", masterModel.getExternalIp());
            LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
            //安装依赖的工具
            cnapSshService.installTools(masterModel);

            //到这一步，说明安装工具完成
            //改状态，通知其他资源
            vmCreateModel.setType(StageTypeEnum.K8S.getType());
            vmCreateModel.setStepOrder(StageDetailTypeEnum.THREE_FOUR.getStepOrder());
            vmCreateModel.setDetailType(StageDetailTypeEnum.THREE_FOUR.getDetailType());
            vmCreateModel.setOrder(StageDetailTypeEnum.THREE_FOUR.getOrder());

            VmCreateModel vmSendCreateModel34 = new VmCreateModel();
            vmResourceService.initVmSendCreateModel(vmSendCreateModel34, vmCreateModel);
            vmResourceService.updateStageMsg(vmSendCreateModel34);
            vmCreateInfoMapper.updateVmCreateInfo(vmCreateModel);
            //结束通知
        }

        if(StageDetailTypeEnum.THREE_FIVE.getStepOrder() > vmCreateModel.getStepOrder()) {
            LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
            LOGGER.warn("installK8sMaster begin no passwd copy k8s ip={}", masterModel.getExternalIp());
            LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
            //配置安装免密的相关环境
            cnapSshService.setNoPassWdCopy(ROOT_NAME, masterModel.getExternalIp(), masterModel.getPasswd(),
                    ROOT_NAME, masterModel.getInnerIp(), masterModel.getPasswd());
            if (!workerModels.isEmpty()) {
                for (VmModel vmModel : workerModels) {

                    //腾讯云做特殊设置
                    if (StringUtils.equals(vmModel.getCloudType(),
                            CloudVendorsEnum.TencentCloud.getCloudType())) {
                        cnapSshService.setTencentRootSsh(vmModel);
                    }

                    //设置免密登录
                    cnapSshService.setNoPassWdCopy(ROOT_NAME,
                            masterModel.getExternalIp(), masterModel.getPasswd(),
                            ROOT_NAME, vmModel.getInnerIp(), vmModel.getPasswd());
                    //设置必须执行的命令
                    cnapSshService.preInstallCmds(vmModel);

                }
            }

            //预先执行的命令
            LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
            LOGGER.warn("preInstallCmds begin cofig k8s ip={}", masterModel.getExternalIp());
            cnapSshService.preInstallCmds(masterModel);

            LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
            LOGGER.warn("installK8sMaster install tools k8s ip={}", masterModel.getExternalIp());
            //设置安装参并且安装
            cnapSshService.installTools(masterModel);

            //到这一步，说明参数配置完成
            //改状态，通知其他资源
            vmCreateModel.setType(StageTypeEnum.K8S.getType());
            vmCreateModel.setStepOrder(StageDetailTypeEnum.THREE_FIVE.getStepOrder());
            vmCreateModel.setDetailType(StageDetailTypeEnum.THREE_FIVE.getDetailType());
            vmCreateModel.setOrder(StageDetailTypeEnum.THREE_FIVE.getOrder());

            VmCreateModel vmSendCreateModel35 = new VmCreateModel();
            vmResourceService.initVmSendCreateModel(vmSendCreateModel35, vmCreateModel);
            vmResourceService.updateStageMsg(vmSendCreateModel35);
            vmCreateInfoMapper.updateVmCreateInfo(vmCreateModel);
            //结束通知
        }

        if (StageDetailTypeEnum.THREE_SIX.getStepOrder() > vmCreateModel.getStepOrder()) {
            //执行命令完成k8s安装
            LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
            LOGGER.warn("installK8sMaster begin cofig k8s ip={}", masterModel.getExternalIp());
            cnapSshService.configInstallParamAndInstall(masterModel, workerModels);

            LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
            LOGGER.warn("installK8sMaster begin getK8sToken k8s ip={}", masterModel.getExternalIp());

            //到这一步，说明k8s安装完成
            //改状态，通知其他资源
            vmCreateModel.setType(StageTypeEnum.K8S.getType());
            vmCreateModel.setStepOrder(StageDetailTypeEnum.THREE_SIX.getStepOrder());
            vmCreateModel.setDetailType(StageDetailTypeEnum.THREE_SIX.getDetailType());
            vmCreateModel.setOrder(StageDetailTypeEnum.THREE_SIX.getOrder());

            VmCreateModel vmSendCreateModel36 = new VmCreateModel();
            vmResourceService.initVmSendCreateModel(vmSendCreateModel36, vmCreateModel);
            vmResourceService.updateStageMsg(vmSendCreateModel36);
            vmCreateInfoMapper.updateVmCreateInfo(vmCreateModel);
            //结束通知
        }

        //获取token
        return cnapSshService.getK8sToken(masterModel);

    }

    @Override
    public void installK8sWorker(String k8sClusterName,VmModel vmModel, VmModel masterModel,VmCreateModel vmCreateModel) throws Exception {

        //安装worker节点
        installK8sWorker(k8sClusterName,vmModel,masterModel,null,vmCreateModel);
        
    }

    @Override
    public void installK8sWorker(String k8sClusterName,VmModel vmModel, VmModel masterModel, String appType,VmCreateModel vmCreateModel) throws Exception {

        LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
        LOGGER.warn("installK8sWorker begin install worker k8s ip={}",vmModel.getInnerIp());

        /**
         * 如果是腾讯云，要对worker节点设置root用户
         */
        //腾讯云特殊处理，如果是腾讯云，那就要设置root用户登陆权限
        if (StringUtils.equals(masterModel.getCloudType(),CloudVendorsEnum.TencentCloud.getCloudType())){
            cnapSshService.setTencentRootSsh(vmModel);
        }

        //防止有历史残留数据，先删除
        cnapSshService.delInnerIpNoPasswdSsh(masterModel, vmModel);

        //通知开始拉取所需资源
        vmCreateModel.setType(StageTypeEnum.K8S.getType());
        vmCreateModel.setStepOrder(StageDetailTypeEnum.THREE_TOW.getStepOrder());
        vmCreateModel.setDetailType(StageDetailTypeEnum.THREE_TOW.getDetailType());
        vmCreateModel.setOrder(StageDetailTypeEnum.THREE_TOW.getOrder());

        VmCreateModel vmSendCreateModel32 = new VmCreateModel();
        vmResourceService.initVmSendCreateModel(vmSendCreateModel32, vmCreateModel);
        vmResourceService.updateStageMsg(vmSendCreateModel32);
        vmCreateInfoMapper.updateVmCreateInfo(vmCreateModel);
        //通知结束

        //做免密登陆
        cnapSshService.setNoPassWdCopy(masterModel.getExternalIp(),masterModel.getPasswd(),ROOT_NAME
                ,vmModel.getInnerIp(),vmModel.getPasswd());
        LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
        LOGGER.warn("installK8sWorker begin preInstallCmds k8s ip={}",vmModel.getInnerIp());
        cnapSshService.preInstallCmds(vmModel);

        //通知做完了配置
        vmCreateModel.setType(StageTypeEnum.K8S.getType());
        vmCreateModel.setStepOrder(StageDetailTypeEnum.THREE_THREE.getStepOrder());
        vmCreateModel.setDetailType(StageDetailTypeEnum.THREE_THREE.getDetailType());
        vmCreateModel.setOrder(StageDetailTypeEnum.THREE_THREE.getOrder());

        VmCreateModel vmSendCreateModel33 = new VmCreateModel();
        vmResourceService.initVmSendCreateModel(vmSendCreateModel33, vmCreateModel);
        vmResourceService.updateStageMsg(vmSendCreateModel33);
        vmCreateInfoMapper.updateVmCreateInfo(vmCreateModel);
        //通知结束

        LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
        LOGGER.warn("installK8sWorker begin addK8sWorker k8s ip={}",vmModel.getInnerIp());
        //安装worker
        cnapSshService.addK8sWorker(masterModel,vmModel);

        //通知安装完毕
        vmCreateModel.setType(StageTypeEnum.K8S.getType());
        vmCreateModel.setStepOrder(StageDetailTypeEnum.THREE_FOUR.getStepOrder());
        vmCreateModel.setDetailType(StageDetailTypeEnum.THREE_FOUR.getDetailType());
        vmCreateModel.setOrder(StageDetailTypeEnum.THREE_FOUR.getOrder());

        VmCreateModel vmSendCreateModel34 = new VmCreateModel();
        vmResourceService.initVmSendCreateModel(vmSendCreateModel34, vmCreateModel);
        vmResourceService.updateStageMsg(vmSendCreateModel34);
        vmCreateInfoMapper.updateVmCreateInfo(vmCreateModel);
        //通知结束

        //woker安装prometheus
        cnapSshService.addWorkerPrometheus(masterModel);

        //通知安装插件完毕
        vmCreateModel.setType(StageTypeEnum.K8S.getType());
        vmCreateModel.setStepOrder(StageDetailTypeEnum.THREE_FIVE.getStepOrder());
        vmCreateModel.setDetailType(StageDetailTypeEnum.THREE_FIVE.getDetailType());
        vmCreateModel.setOrder(StageDetailTypeEnum.THREE_FIVE.getOrder());

        VmCreateModel vmSendCreateModel35 = new VmCreateModel();
        vmResourceService.initVmSendCreateModel(vmSendCreateModel35, vmCreateModel);
        vmResourceService.updateStageMsg(vmSendCreateModel35);
        vmCreateInfoMapper.updateVmCreateInfo(vmCreateModel);
        //通知结束
    }

    @Override
    public K8sClusterModel getK8sClusterModelByZoneIdNoBizId(VmCreateModel vmCreateModel) {
        VmModel vmModel = vmCreateModel.getVmModels().get(0);
        if (StringUtils.isEmpty(vmModel.getZoneId())){
            vmModel.setZoneId(vmModel.getCloudType());
        }
        //私有云只有zoneId没有regionId
        if (StringUtils.isEmpty(vmModel.getRegionId())){
            vmModel.setRegionId(vmModel.getZoneId());
        }

        return k8sClusterInfoMapper.getK8sClusterInfoLikeZone(
                getZoneId(vmModel,vmCreateModel.getUserName()));
    }

    @Override
    public K8sClusterModel getK8sClusterModelByZoneId(VmCreateModel vmCreateModel) {
        VmModel vmModel = vmCreateModel.getVmModels().get(0);
        if (StringUtils.isEmpty(vmModel.getZoneId())){
            vmModel.setZoneId(vmModel.getCloudType());
        }
        //私有云只有zoneId没有regionId
        if (StringUtils.isEmpty(vmModel.getRegionId())){
            vmModel.setRegionId(vmModel.getZoneId());
        }

        return k8sClusterInfoMapper.getK8sClusterInfoByZone(
                getZoneId(vmModel,vmCreateModel.getUserName()));
    }

    @Override
    public List<K8sClusterModel> getK8sClusterInfosByZone(VmModel vmModel,String userName) {
        String zoneId = getZoneIdWithBizId(vmModel,userName);
        return k8sClusterInfoMapper.getK8sClusterInfosByZone(zoneId);
    }

    private String getZoneId(VmModel vmModel, String userName){

        //        if (StringUtils.isNotEmpty(vmModel.getCategory())){
//            zoneId = zoneId + "+" + vmModel.getCategory().toLowerCase();
//        }
        return vmModel.getCloudType() + "+"
                + vmModel.getRegionId() + "+"
                + vmModel.getZoneId() + "+"
                + userName + vmModel.getAccountName() + vmModel.getArch();
    }

    private String getZoneIdWithBizId(VmModel vmModel, String userName){

        //        if (StringUtils.isNotEmpty(vmModel.getCategory())){
//            zoneId = zoneId + "+" + vmModel.getCategory().toLowerCase();
//        }
        return vmModel.getCloudType() + "+"
                + vmModel.getRegionId() + "+"
                + vmModel.getZoneId() + "+"
                + userName + vmModel.getAccountName() + vmModel.getArch() + vmModel.getBusinessId();
    }

    @Override
    public void updateVmJson(K8sClusterBasicModel k8sClusterBasicModel, String vmJson) {
        try{

            k8sClusterInfoMapper.updateClusterVmInfo(k8sClusterBasicModel.getK8sClusterIp(),vmJson);
        }catch (Throwable e){
            LOGGER.error("updateVmJson fail ip={}",k8sClusterBasicModel.getK8sClusterIp(),e);
        }
    }

    /**
     *
     * @param checkDelVmModel
     * @return
     */
    @Override
    public CheckDelVmModel checkVmDelResult(CheckDelVmModel checkDelVmModel) {

        checkDelVmModel.setCheckDel(true);

        //获取cloudeType下所有的机器
        List<K8sClusterModel> k8sClusterModels =
                k8sClusterInfoMapper.getAllK8sClusterByClouderVendor(
                        CloudVendorsEnum.getDescribeByCloudeType(checkDelVmModel.getCloudType()));
        //没有任何集群就返回
        if ((k8sClusterModels == null)||(k8sClusterModels.isEmpty())){
            checkDelVmModel.setCheckDel(true);
            return checkDelVmModel;
        }
        //对比vmId
        for (K8sClusterModel k8sClusterModel:k8sClusterModels){
            for (VmModel vmModel:k8sClusterModel.getVmModels()){
                if (StringUtils.equals(vmModel.getVmId(),checkDelVmModel.getVmId())){
                    checkDelVmModel.setCheckDel(false);
                    return checkDelVmModel;
                }
            }
        }

        return checkDelVmModel;
    }

    @Override
    public void informOtherSvcClusterChange(String clusterName) {

        String url = INFORM_K8S_CHANGE_URL + "/cloud/v1/refresh";

        if (StringUtils.isNotEmpty(clusterName)){
            url = url + "?clusterName=" + clusterName;
        }

        LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
        LOGGER.warn("informOtherSvcClusterChange url={}",url);

        CloseableHttpClient httpclient = HttpClients.createDefault();

        HttpGet httpGet = new HttpGet(url);

        RequestConfig requestConfig = RequestConfig.custom()
                .setConnectTimeout(60 * 1000).setConnectionRequestTimeout(60 * 1000)
                .setSocketTimeout(60 * 1000).build();
        httpGet.setConfig(requestConfig);
        try {

            CloseableHttpResponse response = httpclient.execute(httpGet);
            LOGGER.info("informOtherSvcClusterChange response code={}",response.getStatusLine().getStatusCode());
        }catch (Throwable e){
            LOGGER.error("informOtherSvcClusterChange fail url={}",url,e);
        }
    }

    /**
     * 通知其他服务集群有变化
     */
    @Override
    public void informOtherSvcClusterChange() {

        informOtherSvcClusterChange(null);
    }

    private void checkK8sStatus(KubernetesClient k8sClient) throws Exception {
        // 获取线程池
        ExecutorService es = Executors.newFixedThreadPool(1);

        // Future用于执行多线程的执行结果
        Future<String> future = es.submit(() -> {
            if (k8sClient != null) {
                k8sClient.nodes().list().getItems().size();
            }
            return "success";
        });

        try {
            //超时时间默认，3s
            int overTime = 3;
            if (StringUtils.isNumeric(k8sStatusOverTimeSeconds)){
                overTime = Integer.parseInt(k8sStatusOverTimeSeconds);
            }


            // futrue.get()测试被执行的程序是否能在timeOut时限内返回字符串
            String result = "failed";
            result = future.get(overTime, TimeUnit.SECONDS);//任务处理超时时间设为 1 秒
            if (!StringUtils.equals(result,"success")){
                throw new Exception("get node info time out");
            }
        } finally {
            // 关闭线程池
            es.shutdown();
        }
    }

    @Override
    public List<CloudVendorInfo> getAllCloudVenderAccountInfo(String userName) {

        //根据username筛选，如果userName是admin就返回所有
        List<AccountInfo> accountInfos = flavorInfoMapper.getAllAccount();
        List<CloudVendorInfo> cloudVendorInfos = new ArrayList<>();
        if (!accountInfos.isEmpty()){

            for (AccountInfo accountInfo:accountInfos){

                //cloudType为空，表示加入所有云
                if (StringUtils.isEmpty(accountInfo.getCloudType())){

                    List<CloudVendorsEnum> cloudVendorsEnums = new ArrayList<>();

                    if (accountInfo.isPrivateCloud()){

                        cloudVendorsEnums = CloudVendorsEnum.getAllPrivateClouderVendor();

                    }else{

                        cloudVendorsEnums = CloudVendorsEnum.getAllPublicClouderVendor();
                    }

                    for (CloudVendorsEnum cloudVendorsEnum:cloudVendorsEnums) {

                        if (StringUtils.equalsIgnoreCase(userName,CnapCloudMgmtConstants.ADMIN_USER_NAME)
                            ||StringUtils.equalsIgnoreCase(userName,accountInfo.getUserName())) {
                            cloudVendorInfos.add(new CloudVendorInfo(
                                    cloudVendorsEnum.getDescribe(), cloudVendorsEnum.getCloudType(),
                                    cloudVendorsEnum.getChineseDescribe(), cloudVendorsEnum.getCloudScope(),
                                    accountInfo.getAccountName()));
                        }
                    }

                }else{

                    //每个account只有一个cloudType
                    CloudVendorsEnum cloudVendorsEnum = CloudVendorsEnum.getCloudVendorByCloudType(accountInfo.getCloudType());
                    if (cloudVendorsEnum != null) {

                        if (StringUtils.equalsIgnoreCase(userName,CnapCloudMgmtConstants.ADMIN_USER_NAME)
                                ||StringUtils.equalsIgnoreCase(userName,accountInfo.getUserName())) {
                            cloudVendorInfos.add(new CloudVendorInfo(
                                    cloudVendorsEnum.getDescribe(), cloudVendorsEnum.getCloudType(),
                                    cloudVendorsEnum.getChineseDescribe(), cloudVendorsEnum.getCloudScope(),
                                    accountInfo.getAccountName()));
                        }
                    }
                }
            }
        }

        return cloudVendorInfos;
    }
}
